{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from google.cloud import bigquery\n",
    "import pandas\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.preprocessing import LabelBinarizer\n",
    "from sklearn.preprocessing import OneHotEncoder \n",
    "import pandas as pd\n",
    "from sklearn import preprocessing\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import roc_auc_score, f1_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "import scipy\n",
    "from sklearn.preprocessing import MultiLabelBinarizer\n",
    "from ast import literal_eval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Client creating using default project: ut-goog\n"
     ]
    }
   ],
   "source": [
    "client = bigquery.Client(location=\"US\")\n",
    "print(\"Client creating using default project: {}\".format(client.project))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### This query runs off the same step used to make the clean complaints dataset (combine_tables.sql file output)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>complaint_id</th>\n",
       "      <th>date_received</th>\n",
       "      <th>product</th>\n",
       "      <th>subproduct</th>\n",
       "      <th>issue</th>\n",
       "      <th>subissue</th>\n",
       "      <th>consumer_complaint_narrative</th>\n",
       "      <th>company_public_response</th>\n",
       "      <th>company_name</th>\n",
       "      <th>state</th>\n",
       "      <th>...</th>\n",
       "      <th>date_sent_to_company</th>\n",
       "      <th>company_response_to_consumer</th>\n",
       "      <th>timely_response</th>\n",
       "      <th>consumer_disputed</th>\n",
       "      <th>sentiment_score</th>\n",
       "      <th>sentiment_magnitude</th>\n",
       "      <th>entities</th>\n",
       "      <th>entity_types</th>\n",
       "      <th>entity_sentiment_scores</th>\n",
       "      <th>entity_sentiment_magnitudes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3037829</td>\n",
       "      <td>2018-10-04</td>\n",
       "      <td>Debt collection</td>\n",
       "      <td>Other debt</td>\n",
       "      <td>Other</td>\n",
       "      <td>Other</td>\n",
       "      <td>ERC ONCE AGAIN WENT ON MY CREDIT REPORTS WITH ...</td>\n",
       "      <td>Company can't verify or dispute the facts in t...</td>\n",
       "      <td>ERC</td>\n",
       "      <td>AR</td>\n",
       "      <td>...</td>\n",
       "      <td>2018-10-04</td>\n",
       "      <td>Closed with explanation</td>\n",
       "      <td>True</td>\n",
       "      <td>None</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.2</td>\n",
       "      <td>['ERC', 'CREDIT REPORTS', 'VALIDATION', 'COLLE...</td>\n",
       "      <td>['ORGANIZATION', 'OTHER', 'OTHER', 'OTHER', 'O...</td>\n",
       "      <td>[-0.20000000298023224, -0.10000000149011612, -...</td>\n",
       "      <td>[0.20000000298023224, 0.10000000149011612, 0.3...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2114401</td>\n",
       "      <td>2016-09-15</td>\n",
       "      <td>Debt collection</td>\n",
       "      <td>Other</td>\n",
       "      <td>Other</td>\n",
       "      <td>Other</td>\n",
       "      <td>I have disputed this debt and asked them not t...</td>\n",
       "      <td>Company believes it acted appropriately as aut...</td>\n",
       "      <td>ERC</td>\n",
       "      <td>CA</td>\n",
       "      <td>...</td>\n",
       "      <td>2016-09-15</td>\n",
       "      <td>Closed with non-monetary relief</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>-0.1</td>\n",
       "      <td>0.3</td>\n",
       "      <td>['debt', 'debt', 'letter', 'debt']</td>\n",
       "      <td>['OTHER', 'OTHER', 'WORK_OF_ART', 'OTHER']</td>\n",
       "      <td>[-0.4000000059604645, 0, 0, 0]</td>\n",
       "      <td>[0.4000000059604645, 0, 0, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2890918</td>\n",
       "      <td>2018-04-30</td>\n",
       "      <td>Debt collection</td>\n",
       "      <td>Other debt</td>\n",
       "      <td>Other</td>\n",
       "      <td>Other</td>\n",
       "      <td>I am a victim of identity theft, company XXXX ...</td>\n",
       "      <td>Company believes it acted appropriately as aut...</td>\n",
       "      <td>ERC</td>\n",
       "      <td>GA</td>\n",
       "      <td>...</td>\n",
       "      <td>2018-04-30</td>\n",
       "      <td>Closed with explanation</td>\n",
       "      <td>True</td>\n",
       "      <td>None</td>\n",
       "      <td>-0.9</td>\n",
       "      <td>0.9</td>\n",
       "      <td>['account', 'Account XXXX', 'XXXX XXXX XXXX', ...</td>\n",
       "      <td>['OTHER', 'EVENT', 'ORGANIZATION', 'OTHER', 'P...</td>\n",
       "      <td>[-0.699999988079071, -0.800000011920929, -0.69...</td>\n",
       "      <td>[1.399999976158142, 1.600000023841858, 1.5, 0....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1936042</td>\n",
       "      <td>2016-05-22</td>\n",
       "      <td>Debt collection</td>\n",
       "      <td>Other</td>\n",
       "      <td>Other</td>\n",
       "      <td>Other</td>\n",
       "      <td>I did n't have the money to pay the bills at t...</td>\n",
       "      <td>Company believes it acted appropriately as aut...</td>\n",
       "      <td>ERC</td>\n",
       "      <td>VA</td>\n",
       "      <td>...</td>\n",
       "      <td>2016-05-26</td>\n",
       "      <td>Closed with non-monetary relief</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>-0.1</td>\n",
       "      <td>0.3</td>\n",
       "      <td>['money', 'bills', 'payment process']</td>\n",
       "      <td>['OTHER', 'OTHER', 'OTHER']</td>\n",
       "      <td>[-0.20000000298023224, -0.20000000298023224, 0...</td>\n",
       "      <td>[0.20000000298023224, 0.20000000298023224, 0.1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3108556</td>\n",
       "      <td>2018-12-24</td>\n",
       "      <td>Debt collection</td>\n",
       "      <td>I do not know</td>\n",
       "      <td>Communication tactics</td>\n",
       "      <td>Frequent or repeated calls</td>\n",
       "      <td>The company ERC have been calling my phone eve...</td>\n",
       "      <td>Company believes it acted appropriately as aut...</td>\n",
       "      <td>ERC</td>\n",
       "      <td>NC</td>\n",
       "      <td>...</td>\n",
       "      <td>2018-12-24</td>\n",
       "      <td>Closed with explanation</td>\n",
       "      <td>True</td>\n",
       "      <td>None</td>\n",
       "      <td>-0.8</td>\n",
       "      <td>3.2</td>\n",
       "      <td>['ERC', 'phone', 'times', 'number', 'XXXX XXXX...</td>\n",
       "      <td>['ORGANIZATION', 'CONSUMER_GOOD', 'EVENT', 'OT...</td>\n",
       "      <td>[-0.5, -0.30000001192092896, -0.60000002384185...</td>\n",
       "      <td>[3.299999952316284, 0.30000001192092896, 0.600...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   complaint_id date_received          product     subproduct  \\\n",
       "0       3037829    2018-10-04  Debt collection     Other debt   \n",
       "1       2114401    2016-09-15  Debt collection          Other   \n",
       "2       2890918    2018-04-30  Debt collection     Other debt   \n",
       "3       1936042    2016-05-22  Debt collection          Other   \n",
       "4       3108556    2018-12-24  Debt collection  I do not know   \n",
       "\n",
       "                   issue                    subissue  \\\n",
       "0                  Other                       Other   \n",
       "1                  Other                       Other   \n",
       "2                  Other                       Other   \n",
       "3                  Other                       Other   \n",
       "4  Communication tactics  Frequent or repeated calls   \n",
       "\n",
       "                        consumer_complaint_narrative  \\\n",
       "0  ERC ONCE AGAIN WENT ON MY CREDIT REPORTS WITH ...   \n",
       "1  I have disputed this debt and asked them not t...   \n",
       "2  I am a victim of identity theft, company XXXX ...   \n",
       "3  I did n't have the money to pay the bills at t...   \n",
       "4  The company ERC have been calling my phone eve...   \n",
       "\n",
       "                             company_public_response company_name state  ...  \\\n",
       "0  Company can't verify or dispute the facts in t...          ERC    AR  ...   \n",
       "1  Company believes it acted appropriately as aut...          ERC    CA  ...   \n",
       "2  Company believes it acted appropriately as aut...          ERC    GA  ...   \n",
       "3  Company believes it acted appropriately as aut...          ERC    VA  ...   \n",
       "4  Company believes it acted appropriately as aut...          ERC    NC  ...   \n",
       "\n",
       "  date_sent_to_company     company_response_to_consumer timely_response  \\\n",
       "0           2018-10-04          Closed with explanation            True   \n",
       "1           2016-09-15  Closed with non-monetary relief            True   \n",
       "2           2018-04-30          Closed with explanation            True   \n",
       "3           2016-05-26  Closed with non-monetary relief            True   \n",
       "4           2018-12-24          Closed with explanation            True   \n",
       "\n",
       "  consumer_disputed sentiment_score sentiment_magnitude  \\\n",
       "0              None             0.0                 0.2   \n",
       "1             False            -0.1                 0.3   \n",
       "2              None            -0.9                 0.9   \n",
       "3             False            -0.1                 0.3   \n",
       "4              None            -0.8                 3.2   \n",
       "\n",
       "                                            entities  \\\n",
       "0  ['ERC', 'CREDIT REPORTS', 'VALIDATION', 'COLLE...   \n",
       "1                 ['debt', 'debt', 'letter', 'debt']   \n",
       "2  ['account', 'Account XXXX', 'XXXX XXXX XXXX', ...   \n",
       "3              ['money', 'bills', 'payment process']   \n",
       "4  ['ERC', 'phone', 'times', 'number', 'XXXX XXXX...   \n",
       "\n",
       "                                        entity_types  \\\n",
       "0  ['ORGANIZATION', 'OTHER', 'OTHER', 'OTHER', 'O...   \n",
       "1         ['OTHER', 'OTHER', 'WORK_OF_ART', 'OTHER']   \n",
       "2  ['OTHER', 'EVENT', 'ORGANIZATION', 'OTHER', 'P...   \n",
       "3                        ['OTHER', 'OTHER', 'OTHER']   \n",
       "4  ['ORGANIZATION', 'CONSUMER_GOOD', 'EVENT', 'OT...   \n",
       "\n",
       "                             entity_sentiment_scores  \\\n",
       "0  [-0.20000000298023224, -0.10000000149011612, -...   \n",
       "1                     [-0.4000000059604645, 0, 0, 0]   \n",
       "2  [-0.699999988079071, -0.800000011920929, -0.69...   \n",
       "3  [-0.20000000298023224, -0.20000000298023224, 0...   \n",
       "4  [-0.5, -0.30000001192092896, -0.60000002384185...   \n",
       "\n",
       "                         entity_sentiment_magnitudes  \n",
       "0  [0.20000000298023224, 0.10000000149011612, 0.3...  \n",
       "1                      [0.4000000059604645, 0, 0, 0]  \n",
       "2  [1.399999976158142, 1.600000023841858, 1.5, 0....  \n",
       "3  [0.20000000298023224, 0.20000000298023224, 0.1...  \n",
       "4  [3.299999952316284, 0.30000001192092896, 0.600...  \n",
       "\n",
       "[5 rows x 24 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#query = \"\"\"\n",
    "#    SELECT \n",
    "#        *\n",
    "#    FROM \n",
    "#        `ut-goog.nl_api.Karan_rolled_up_data`\n",
    "#\"\"\"\n",
    "#query_job = client.query(\n",
    "#    query,\n",
    "    # Location must match that of the dataset(s) referenced in the query.\n",
    "#    location=\"US\",\n",
    "#)  # API request - starts the query\n",
    "\n",
    "#complaints_df = query_job.to_dataframe()\n",
    "#complaints_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# One Hot Encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "complaints_df['product'] = complaints_df['product'].astype('category')\n",
    "complaints_df['subproduct'] = complaints_df['subproduct'].astype('category')\n",
    "complaints_df['issue'] = complaints_df['issue'].astype('category')\n",
    "complaints_df['subissue'] = complaints_df['subissue'].astype('category')\n",
    "complaints_df['state'] = complaints_df['state'].astype('category')\n",
    "complaints_df['zip_code'] = complaints_df['zip_code'].astype('category')\n",
    "complaints_df['tags'] = complaints_df['tags'].astype('category')\n",
    "\n",
    "variables = ['company_response_to_consumer',\n",
    "             'product',\n",
    "             'subproduct',\n",
    "             'issue',\n",
    "             'subissue',\n",
    "             'state',\n",
    "             'zip_code',\n",
    "             'tags',\n",
    "             'consumer_complaint_narrative']\n",
    "complaints_df2 = complaints_df.copy()\n",
    "complaints_df2 = complaints_df2[variables]\n",
    "complaints_df2.dropna(subset=['company_response_to_consumer'],inplace=True)\n",
    "\n",
    "one_hot1 = pd.get_dummies(complaints_df2['product'], drop_first=True)\n",
    "one_hot2 = pd.get_dummies(complaints_df2['subproduct'], drop_first=True)\n",
    "one_hot3 = pd.get_dummies(complaints_df2['issue'], drop_first=True)\n",
    "one_hot4 = pd.get_dummies(complaints_df2['subissue'], drop_first=True)\n",
    "one_hot5 = pd.get_dummies(complaints_df2['state'], drop_first=True)\n",
    "one_hot6 = pd.get_dummies(complaints_df2['zip_code'], drop_first=True)\n",
    "one_hot7 = pd.get_dummies(complaints_df2['tags'], drop_first=True)\n",
    "\n",
    "complaints_df2 = complaints_df2.join(one_hot1, rsuffix='prod')\n",
    "complaints_df2 = complaints_df2.join(one_hot2, rsuffix='subprod')\n",
    "complaints_df2 = complaints_df2.join(one_hot3, rsuffix='issue')\n",
    "complaints_df2 = complaints_df2.join(one_hot4, rsuffix='subissue')\n",
    "complaints_df2 = complaints_df2.join(one_hot5)\n",
    "#complaints_df2 = complaints_df2.join(one_hot6)\n",
    "complaints_df2 = complaints_df2.join(one_hot7)\n",
    "\n",
    "variable = ['company_response_to_consumer','product','subproduct','issue','subissue','state','zip_code','tags']\n",
    "\n",
    "X = complaints_df2.drop(columns=variable)\n",
    "y = complaints_df2['company_response_to_consumer']\n",
    "#labels = y.unique()\n",
    "#y = preprocessing.label_binarize(y, classes=labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Random Forest: No Text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=11)\n",
    "X_train = X_train.drop(columns=['consumer_complaint_narrative'])\n",
    "X_test = X_test.drop(columns=['consumer_complaint_narrative'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'cv': None,\n",
       " 'error_score': 'raise',\n",
       " 'estimator': RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "             max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "             min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "             min_samples_leaf=1, min_samples_split=2,\n",
       "             min_weight_fraction_leaf=0.0, n_estimators=10, n_jobs=1,\n",
       "             oob_score=False, random_state=None, verbose=0,\n",
       "             warm_start=False),\n",
       " 'estimator__bootstrap': True,\n",
       " 'estimator__class_weight': None,\n",
       " 'estimator__criterion': 'gini',\n",
       " 'estimator__max_depth': None,\n",
       " 'estimator__max_features': 'auto',\n",
       " 'estimator__max_leaf_nodes': None,\n",
       " 'estimator__min_impurity_decrease': 0.0,\n",
       " 'estimator__min_impurity_split': None,\n",
       " 'estimator__min_samples_leaf': 1,\n",
       " 'estimator__min_samples_split': 2,\n",
       " 'estimator__min_weight_fraction_leaf': 0.0,\n",
       " 'estimator__n_estimators': 10,\n",
       " 'estimator__n_jobs': 1,\n",
       " 'estimator__oob_score': False,\n",
       " 'estimator__random_state': None,\n",
       " 'estimator__verbose': 0,\n",
       " 'estimator__warm_start': False,\n",
       " 'fit_params': None,\n",
       " 'iid': True,\n",
       " 'n_jobs': 1,\n",
       " 'param_grid': {'n_estimators': [10, 100, 200, 300], 'n_jobs': [-1]},\n",
       " 'pre_dispatch': '2*n_jobs',\n",
       " 'refit': True,\n",
       " 'return_train_score': 'warn',\n",
       " 'scoring': None,\n",
       " 'verbose': 0}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "parameters = {#'class_weight':[None,'balanced'],\n",
    "              #'max_depth':[2,4,6,8],\n",
    "              'n_estimators':[10,100,200,300],\n",
    "              'n_jobs':[-1]\n",
    "             }\n",
    "rf = RandomForestClassifier()\n",
    "clf = GridSearchCV(rf, parameters)\n",
    "clf.get_params()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'n_estimators': 300, 'n_jobs': -1}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training accuracy: 0.8115 \n",
      "test accuracy: 0.8049 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "y_pred = clf.predict(X_train)\n",
    "y_pred_test = clf.predict(X_test)\n",
    "y_score_test = clf.predict_proba(X_test)[:,1]\n",
    "print(\"training accuracy:\", round(metrics.accuracy_score(y_train,y_pred),4), \"\\n\"\n",
    "     \"test accuracy:\", round(metrics.accuracy_score(y_test,y_pred_test),4), \"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[    0,   742,     2,     5,     0],\n",
       "       [   11, 79781,   140,   318,    13],\n",
       "       [    0,  5299,    52,    20,     0],\n",
       "       [    2, 12205,    22,   132,     0],\n",
       "       [    1,   594,     1,     3,     1]])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test,y_pred_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def multiclass_roc_auc_score(truth, pred, average=\"macro\"):\n",
    "\n",
    "    lb = LabelBinarizer()\n",
    "    lb.fit(truth)\n",
    "\n",
    "    truth = lb.transform(truth)\n",
    "    pred = lb.transform(pred)\n",
    "\n",
    "    return roc_auc_score(truth, pred, average=average)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "multiclass auroc: 0.502264488460064\n",
      "f1 score: 0.7242465439128403\n"
     ]
    }
   ],
   "source": [
    "print('multiclass auroc:',multiclass_roc_auc_score(y_test,y_pred_test))\n",
    "print('f1 score:', f1_score(y_test, y_pred_test, average='weighted'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Naive Bayes w/ no text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "nb = MultinomialNB()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training accuracy: 0.6832 \n",
      "test accuracy: 0.6811 \n",
      "multiclass aucroc: 0.5951 \n",
      "f1 score: 0.696 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "nb.fit(X_train, y_train)\n",
    "\n",
    "y_pred = nb.predict(X_train)\n",
    "y_pred_test = nb.predict(X_test)\n",
    "y_score_test = nb.predict_proba(X_test)[:,1]\n",
    "print(\"training accuracy:\", round(metrics.accuracy_score(y_train,y_pred),4), \"\\n\"\n",
    "     \"test accuracy:\", round(metrics.accuracy_score(y_test,y_pred_test),4), \"\\n\"\n",
    "     \"multiclass aucroc:\", round(multiclass_roc_auc_score(y_test,y_pred_test),4), '\\n'\n",
    "     \"f1 score:\",round(f1_score(y_test, y_pred_test, average='weighted'),4),'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  204,   516,    20,     5,     4],\n",
       "       [ 2757, 62344,  9314,  5655,   193],\n",
       "       [   31,  2330,  3002,     7,     1],\n",
       "       [  488,  8542,  1166,  2106,    59],\n",
       "       [   45,   519,    17,    13,     6]])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test,y_pred_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Random Forest with traditional methods"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_text = complaints_df2.drop(columns=variable)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_text, y, test_size=0.2, random_state=11)\n",
    "train_text = X_train['consumer_complaint_narrative']\n",
    "test_text = X_test['consumer_complaint_narrative']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "count_vect = CountVectorizer(analyzer='word', token_pattern=r'\\w{1,}')\n",
    "count_vect.fit(complaints_df['consumer_complaint_narrative'])\n",
    "\n",
    "xtrain_count =  count_vect.transform(train_text)\n",
    "xtest_count =  count_vect.transform(test_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<397373x132147 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 37102072 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xtrain_count"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### With Meta Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train2 = X_train.drop(columns=['consumer_complaint_narrative'])\n",
    "X_test2 = X_test.drop(columns=['consumer_complaint_narrative'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_counts = scipy.sparse.hstack([xtrain_count, X_train2])\n",
    "X_test_counts = scipy.sparse.hstack([xtest_count, X_test2])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note that using the grid search will work on the following random forests, but will take a long time per random forest (several hours)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "#rfc = RandomForestClassifier(n_estimators = 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training accuracy: 0.962 \n",
      "test accuracy: 0.796 \n",
      "multiclass aucroc: 0.5185 \n",
      "f1 score: 0.7363 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "clf.fit(X_train_counts, y_train)\n",
    "y_pred = clf.predict(X_train_counts)\n",
    "y_pred_test = clf.predict(X_test_counts)\n",
    "y_score_test = clf.predict_proba(X_test_counts)[:,1]\n",
    "print(\"training accuracy:\", round(metrics.accuracy_score(y_train,y_pred),4), \"\\n\"\n",
    "     \"test accuracy:\", round(metrics.accuracy_score(y_test,y_pred_test),4), \"\\n\"\n",
    "     \"multiclass aucroc:\", round(multiclass_roc_auc_score(y_test,y_pred_test),4), '\\n'\n",
    "     \"f1 score:\",round(f1_score(y_test, y_pred_test, average='weighted'),4),'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'n_estimators': 10, 'n_jobs': -1}"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Seems like it did not need many trees. Will reduce for the next ones."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "parameters = {#'class_weight':[None,'balanced'],\n",
    "              #'max_depth':[2,4,6,8],\n",
    "              'n_estimators':[10,50,100],\n",
    "              'n_jobs':[-1]\n",
    "             }\n",
    "rf = RandomForestClassifier()\n",
    "clf = GridSearchCV(rf, parameters)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Just Text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training accuracy: 0.9605 \n",
      "test accuracy: 0.7959 \n",
      "multiclass aucroc: 0.5143 \n",
      "f1 score: 0.733 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "clf.fit(xtrain_count, y_train)\n",
    "y_pred = clf.predict(xtrain_count)\n",
    "y_pred_test = clf.predict(xtest_count)\n",
    "y_score_test = clf.predict_proba(xtest_count)[:,1]\n",
    "print(\"training accuracy:\", round(metrics.accuracy_score(y_train,y_pred),4), \"\\n\"\n",
    "     \"test accuracy:\", round(metrics.accuracy_score(y_test,y_pred_test),4), \"\\n\"\n",
    "    \"multiclass aucroc:\", round(multiclass_roc_auc_score(y_test,y_pred_test),4), '\\n'\n",
    "     \"f1 score:\",round(f1_score(y_test, y_pred_test, average='weighted'),4),'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[   22,   723,     0,     4,     0],\n",
       "       [   27, 77996,   125,  2071,    44],\n",
       "       [    0,  5259,    94,    18,     0],\n",
       "       [    1, 11387,    19,   950,     4],\n",
       "       [    0,   583,     0,     8,     9]])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test,y_pred_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'n_estimators': 10, 'n_jobs': -1}"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Naive Bayes Using Counts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Counts + meta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training accuracy: 0.5364 \n",
      "test accuracy: 0.5282 \n",
      "multiclass aucroc: 0.6309 \n",
      "f1 score: 0.5809 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "nb.fit(X_train_counts, y_train)\n",
    "y_pred = nb.predict(X_train_counts)\n",
    "y_pred_test = nb.predict(X_test_counts)\n",
    "y_score_test = nb.predict_proba(X_test_counts)[:,1]\n",
    "print(\"training accuracy:\", round(metrics.accuracy_score(y_train,y_pred),4), \"\\n\"\n",
    "     \"test accuracy:\", round(metrics.accuracy_score(y_test,y_pred_test),4), \"\\n\"\n",
    "     \"multiclass aucroc:\", round(multiclass_roc_auc_score(y_test,y_pred_test),4), '\\n'\n",
    "     \"f1 score:\",round(f1_score(y_test, y_pred_test, average='weighted'),4),'\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Just counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training accuracy: 0.5661 \n",
      "test accuracy: 0.5576 \n",
      "multiclass aucroc: 0.6222 \n",
      "f1 score: 0.6078 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "nb.fit(xtrain_count, y_train)\n",
    "y_pred = nb.predict(xtrain_count)\n",
    "y_pred_test = nb.predict(xtest_count)\n",
    "y_score_test = nb.predict_proba(xtest_count)[:,1]\n",
    "print(\"training accuracy:\", round(metrics.accuracy_score(y_train,y_pred),4), \"\\n\"\n",
    "     \"test accuracy:\", round(metrics.accuracy_score(y_test,y_pred_test),4), \"\\n\"\n",
    "      \"multiclass aucroc:\", round(multiclass_roc_auc_score(y_test,y_pred_test),4), '\\n'\n",
    "     \"f1 score:\",round(f1_score(y_test, y_pred_test, average='weighted'),4),'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[   26,   515,    79,   110,    19],\n",
       "       [   91, 44505, 12192, 23193,   282],\n",
       "       [    1,  1017,  4246,   103,     4],\n",
       "       [    4,  4363,  1374,  6583,    37],\n",
       "       [    0,   432,    41,    90,    37]])"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test,y_pred_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Random Forest/Naive Bayes Using TF-IDF"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## unigram-tf-idf: random forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf_vect = TfidfVectorizer(analyzer='word', token_pattern=r'\\w{1,}', max_features=5000)\n",
    "tfidf_vect.fit(complaints_df['consumer_complaint_narrative'])\n",
    "xtrain_tfidf =  tfidf_vect.transform(train_text)\n",
    "xtest_tfidf =  tfidf_vect.transform(test_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_tfidf = scipy.sparse.hstack([xtrain_tfidf, X_train2])\n",
    "X_test_tfidf = scipy.sparse.hstack([xtest_tfidf, X_test2])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### With text and meta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training accuracy: 0.9848 \n",
      "test accuracy: 0.7991 \n",
      "multiclass aucroc: 0.5172 \n",
      "f1 score: 0.7367 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "clf.fit(X_train_tfidf, y_train)\n",
    "y_pred = clf.predict(X_train_tfidf)\n",
    "y_pred_test = clf.predict(X_test_tfidf)\n",
    "y_score_test = clf.predict_proba(X_test_tfidf)[:,1]\n",
    "print(\"training accuracy:\", round(metrics.accuracy_score(y_train,y_pred),4), \"\\n\"\n",
    "     \"test accuracy:\", round(metrics.accuracy_score(y_test,y_pred_test),4), \"\\n\"\n",
    "     \"multiclass aucroc:\", round(multiclass_roc_auc_score(y_test,y_pred_test),4), '\\n'\n",
    "     \"f1 score:\",round(f1_score(y_test, y_pred_test, average='weighted'),4),'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[   25,   719,     0,     5,     0],\n",
       "       [   22, 78174,    86,  1933,    48],\n",
       "       [    0,  5227,   124,    20,     0],\n",
       "       [    1, 11292,    15,  1050,     3],\n",
       "       [    0,   582,     0,     9,     9]])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test,y_pred_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Just text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training accuracy: 0.9838 \n",
      "test accuracy: 0.7984 \n",
      "multiclass aucroc: 0.5153 \n",
      "f1 score: 0.7344 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "clf.fit(xtrain_tfidf, y_train)\n",
    "y_pred = clf.predict(xtrain_tfidf)\n",
    "y_pred_test = clf.predict(xtest_tfidf)\n",
    "y_score_test = clf.predict_proba(xtest_tfidf)[:,1]\n",
    "print(\"training accuracy:\", round(metrics.accuracy_score(y_train,y_pred),4), \"\\n\"\n",
    "     \"test accuracy:\", round(metrics.accuracy_score(y_test,y_pred_test),4), \"\\n\"\n",
    "      \"multiclass aucroc:\", round(multiclass_roc_auc_score(y_test,y_pred_test),4), '\\n'\n",
    "     \"f1 score:\",round(f1_score(y_test, y_pred_test, average='weighted'),4),'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[   26,   718,     0,     5,     0],\n",
       "       [   22, 78244,    79,  1875,    43],\n",
       "       [    0,  5263,    90,    18,     0],\n",
       "       [    2, 11396,    12,   947,     4],\n",
       "       [    0,   583,     0,     8,     9]])"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test,y_pred_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## unigram-tf-idf: naive bayes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Text and meta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training accuracy: 0.6398 \n",
      "test accuracy: 0.636 \n",
      "multiclass aucroc: 0.636 \n",
      "f1 score: 0.6734 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "nb.fit(X_train_tfidf, y_train)\n",
    "y_pred = nb.predict(X_train_tfidf)\n",
    "y_pred_test = nb.predict(X_test_tfidf)\n",
    "y_score_test = nb.predict_proba(X_test_tfidf)[:,1]\n",
    "print(\"training accuracy:\", round(metrics.accuracy_score(y_train,y_pred),4), \"\\n\"\n",
    "     \"test accuracy:\", round(metrics.accuracy_score(y_test,y_pred_test),4), \"\\n\"\n",
    "     \"multiclass aucroc:\", round(multiclass_roc_auc_score(y_test,y_pred_test),4), '\\n'\n",
    "     \"f1 score:\",round(f1_score(y_test, y_pred_test, average='weighted'),4),'\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Just with the text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training accuracy: 0.8078 \n",
      "test accuracy: 0.8068 \n",
      "multiclass aucroc: 0.5191 \n",
      "f1 score: 0.7361 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/site-packages/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "nb.fit(xtrain_tfidf, y_train)\n",
    "y_pred = nb.predict(xtrain_tfidf)\n",
    "y_pred_test = nb.predict(xtest_tfidf)\n",
    "y_score_test = nb.predict_proba(xtest_tfidf)[:,1]\n",
    "print(\"training accuracy:\", round(metrics.accuracy_score(y_train,y_pred),4), \"\\n\"\n",
    "     \"test accuracy:\", round(metrics.accuracy_score(y_test,y_pred_test),4), \"\\n\"\n",
    "     \"multiclass aucroc:\", round(multiclass_roc_auc_score(y_test,y_pred_test),4), '\\n'\n",
    "     \"f1 score:\",round(f1_score(y_test, y_pred_test, average='weighted'),4),'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[   15,   728,     3,     3,     0],\n",
       "       [    0, 79261,   689,   313,     0],\n",
       "       [    0,  4698,   673,     0,     0],\n",
       "       [    0, 12029,   128,   204,     0],\n",
       "       [    0,   600,     0,     0,     0]])"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test,y_pred_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## bi-grams: random forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf_vect_bigram = TfidfVectorizer(analyzer='word', token_pattern=r'\\w{1,}', ngram_range=(2,2), max_features=5000)\n",
    "tfidf_vect_bigram.fit(complaints_df['consumer_complaint_narrative'])\n",
    "xtrain_tfidf_bigram =  tfidf_vect_bigram.transform(train_text)\n",
    "xvalid_tfidf_bigram =  tfidf_vect_bigram.transform(test_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_tfidf_bi = scipy.sparse.hstack([xtrain_tfidf_bigram, X_train2])\n",
    "X_test_tfidf_bi = scipy.sparse.hstack([xvalid_tfidf_bigram, X_test2])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Text and Meta Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training accuracy: 0.9841 \n",
      "test accuracy: 0.7969 \n",
      "multiclass aucroc: 0.5166 \n",
      "f1 score: 0.7353 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "clf.fit(X_train_tfidf_bi, y_train)\n",
    "y_pred = clf.predict(X_train_tfidf_bi)\n",
    "y_pred_test = clf.predict(X_test_tfidf_bi)\n",
    "y_score_test = clf.predict_proba(X_test_tfidf_bi)[:,1]\n",
    "print(\"training accuracy:\", round(metrics.accuracy_score(y_train,y_pred),4), \"\\n\"\n",
    "     \"test accuracy:\", round(metrics.accuracy_score(y_test,y_pred_test),4), \"\\n\"\n",
    "     \"multiclass aucroc:\", round(multiclass_roc_auc_score(y_test,y_pred_test),4), '\\n'\n",
    "     \"f1 score:\",round(f1_score(y_test, y_pred_test, average='weighted'),4),'\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Just text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training accuracy: 0.9826 \n",
      "test accuracy: 0.7963 \n",
      "multiclass aucroc: 0.5138 \n",
      "f1 score: 0.7326 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "clf.fit(xtrain_tfidf_bigram, y_train)\n",
    "y_pred = clf.predict(xtrain_tfidf_bigram)\n",
    "y_pred_test = clf.predict(xvalid_tfidf_bigram)\n",
    "y_score_test = clf.predict_proba(xvalid_tfidf_bigram)[:,1]\n",
    "print(\"training accuracy:\", round(metrics.accuracy_score(y_train,y_pred),4), \"\\n\"\n",
    "     \"test accuracy:\", round(metrics.accuracy_score(y_test,y_pred_test),4), \"\\n\"\n",
    "     \"multiclass aucroc:\", round(multiclass_roc_auc_score(y_test,y_pred_test),4), '\\n'\n",
    "     \"f1 score:\",round(f1_score(y_test, y_pred_test, average='weighted'),4),'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[   20,   727,     0,     2,     0],\n",
       "       [   30, 78067,    82,  2032,    52],\n",
       "       [    0,  5271,    90,    10,     0],\n",
       "       [    1, 11433,     3,   921,     3],\n",
       "       [    0,   579,     0,    11,    10]])"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test,y_pred_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## bi-grams: naive bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training accuracy: 0.6049 \n",
      "test accuracy: 0.6032 \n",
      "multiclass aucroc: 0.6421 \n",
      "f1 score: 0.65 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "nb.fit(X_train_tfidf_bi, y_train)\n",
    "y_pred = nb.predict(X_train_tfidf_bi)\n",
    "y_pred_test = nb.predict(X_test_tfidf_bi)\n",
    "y_score_test = nb.predict_proba(X_test_tfidf_bi)[:,1]\n",
    "print(\"training accuracy:\", round(metrics.accuracy_score(y_train,y_pred),4), \"\\n\"\n",
    "     \"test accuracy:\", round(metrics.accuracy_score(y_test,y_pred_test),4), \"\\n\"\n",
    "     \"multiclass aucroc:\", round(multiclass_roc_auc_score(y_test,y_pred_test),4), '\\n'\n",
    "     \"f1 score:\",round(f1_score(y_test, y_pred_test, average='weighted'),4),'\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Just text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training accuracy: 0.7989 \n",
      "test accuracy: 0.7985 \n",
      "multiclass aucroc: 0.5322 \n",
      "f1 score: 0.7422 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/site-packages/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "nb.fit(xtrain_tfidf_bigram, y_train)\n",
    "y_pred = nb.predict(xtrain_tfidf_bigram)\n",
    "y_pred_test = nb.predict(xvalid_tfidf_bigram)\n",
    "y_score_test = nb.predict_proba(xvalid_tfidf_bigram)[:,1]\n",
    "print(\"training accuracy:\", round(metrics.accuracy_score(y_train,y_pred),4), \"\\n\"\n",
    "     \"test accuracy:\", round(metrics.accuracy_score(y_test,y_pred_test),4), \"\\n\"\n",
    "     \"multiclass aucroc:\", round(multiclass_roc_auc_score(y_test,y_pred_test),4), '\\n'\n",
    "     \"f1 score:\",round(f1_score(y_test, y_pred_test, average='weighted'),4),'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[    0,   735,    11,     3,     0],\n",
       "       [    0, 77511,  1673,  1067,    12],\n",
       "       [    0,  4102,  1268,     1,     0],\n",
       "       [    0, 11641,   173,   547,     0],\n",
       "       [    0,   597,     0,     1,     2]])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test,y_pred_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## bi/tri grams: random forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf_vect_ngram = TfidfVectorizer(analyzer='word', token_pattern=r'\\w{1,}', ngram_range=(2,3), max_features=5000)\n",
    "tfidf_vect_ngram.fit(complaints_df['consumer_complaint_narrative'])\n",
    "xtrain_tfidf_ngram =  tfidf_vect_ngram.transform(train_text)\n",
    "xvalid_tfidf_ngram =  tfidf_vect_ngram.transform(test_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_tfidf_n = scipy.sparse.hstack([xtrain_tfidf_ngram, X_train2])\n",
    "X_test_tfidf_n = scipy.sparse.hstack([xvalid_tfidf_ngram, X_test2])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### with meta and text data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training accuracy: 0.9824 \n",
      "test accuracy: 0.7972 \n",
      "multiclass aucroc: 0.5137 \n",
      "f1 score: 0.7329 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "clf.fit(xtrain_tfidf_ngram, y_train)\n",
    "y_pred = clf.predict(xtrain_tfidf_ngram)\n",
    "y_pred_test = clf.predict(xvalid_tfidf_ngram)\n",
    "y_score_test = clf.predict_proba(xvalid_tfidf_ngram)[:,1]\n",
    "print(\"training accuracy:\", round(metrics.accuracy_score(y_train,y_pred),4), \"\\n\"\n",
    "     \"test accuracy:\", round(metrics.accuracy_score(y_test,y_pred_test),4), \"\\n\"\n",
    "     \"multiclass aucroc:\", round(multiclass_roc_auc_score(y_test,y_pred_test),4), '\\n'\n",
    "     \"f1 score:\",round(f1_score(y_test, y_pred_test, average='weighted'),4),'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[   18,   727,     1,     3,     0],\n",
       "       [   24, 78178,    80,  1933,    48],\n",
       "       [    0,  5260,   101,    10,     0],\n",
       "       [    1, 11460,     6,   891,     3],\n",
       "       [    0,   580,     0,    10,    10]])"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test,y_pred_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Just text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training accuracy: 0.9823 \n",
      "test accuracy: 0.7974 \n",
      "multiclass aucroc: 0.5135 \n",
      "f1 score: 0.7328 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "clf.fit(xtrain_tfidf_ngram, y_train)\n",
    "y_pred = clf.predict(xtrain_tfidf_ngram)\n",
    "y_pred_test = clf.predict(xvalid_tfidf_ngram)\n",
    "y_score_test = clf.predict_proba(xvalid_tfidf_ngram)[:,1]\n",
    "print(\"training accuracy:\", round(metrics.accuracy_score(y_train,y_pred),4), \"\\n\"\n",
    "     \"test accuracy:\", round(metrics.accuracy_score(y_test,y_pred_test),4), \"\\n\"\n",
    "     \"multiclass aucroc:\", round(multiclass_roc_auc_score(y_test,y_pred_test),4), '\\n'\n",
    "     \"f1 score:\",round(f1_score(y_test, y_pred_test, average='weighted'),4),'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[   20,   725,     1,     3,     0],\n",
       "       [   21, 78207,    78,  1911,    46],\n",
       "       [    0,  5275,    91,     5,     0],\n",
       "       [    1, 11466,     3,   888,     3],\n",
       "       [    0,   582,     0,     9,     9]])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test,y_pred_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## bi/tri grams: naive bayes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Text + Meta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training accuracy: 0.5945 \n",
      "test accuracy: 0.592 \n",
      "multiclass aucroc: 0.6416 \n",
      "f1 score: 0.6414 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "nb.fit(X_train_tfidf_n, y_train)\n",
    "y_pred = nb.predict(X_train_tfidf_n)\n",
    "y_pred_test = nb.predict(X_test_tfidf_n)\n",
    "y_score_test = nb.predict_proba(X_test_tfidf_n)[:,1]\n",
    "print(\"training accuracy:\", round(metrics.accuracy_score(y_train,y_pred),4), \"\\n\"\n",
    "     \"test accuracy:\", round(metrics.accuracy_score(y_test,y_pred_test),4), \"\\n\"\n",
    "     \"multiclass aucroc:\", round(multiclass_roc_auc_score(y_test,y_pred_test),4), '\\n'\n",
    "     \"f1 score:\",round(f1_score(y_test, y_pred_test, average='weighted'),4),'\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Just text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training accuracy: 0.7892 \n",
      "test accuracy: 0.7892 \n",
      "multiclass aucroc: 0.5392 \n",
      "f1 score: 0.7426 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/site-packages/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "nb.fit(xtrain_tfidf_ngram, y_train)\n",
    "y_pred = nb.predict(xtrain_tfidf_ngram)\n",
    "y_pred_test = nb.predict(xvalid_tfidf_ngram)\n",
    "y_score_test = nb.predict_proba(xvalid_tfidf_ngram)[:,1]\n",
    "print(\"training accuracy:\", round(metrics.accuracy_score(y_train,y_pred),4), \"\\n\"\n",
    "     \"test accuracy:\", round(metrics.accuracy_score(y_test,y_pred_test),4), \"\\n\"\n",
    "     \"multiclass aucroc:\", round(multiclass_roc_auc_score(y_test,y_pred_test),4), '\\n'\n",
    "     \"f1 score:\",round(f1_score(y_test, y_pred_test, average='weighted'),4),'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[    0,   728,    16,     5,     0],\n",
       "       [    0, 76042,  2412,  1794,    15],\n",
       "       [    0,  3868,  1501,     2,     0],\n",
       "       [    0, 11244,   259,   858,     0],\n",
       "       [    0,   593,     2,     1,     4]])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test,y_pred_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Using data from NLP-API\n",
    "\n",
    "Link to the docs: https://cloud.google.com/natural-language/docs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The NLP-API added sentiment_score (of the consumer_complaint_narrative), mag (complaint sentiment magnitude), ent (extracted entities), ent types (the type of the extracted entities), entity_sent_scores (entities sentiment scores), entity_sent_mag (entities sentiment scores magnitude)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "mlb = MultiLabelBinarizer(sparse_output=True)\n",
    "complaints_df3 = complaints_df.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "complaints_df3.dropna(subset=['company_response_to_consumer'],inplace=True)\n",
    "\n",
    "one_hot1 = pd.get_dummies(complaints_df3['product'], drop_first=True)\n",
    "one_hot2 = pd.get_dummies(complaints_df3['subproduct'], drop_first=True)\n",
    "one_hot3 = pd.get_dummies(complaints_df3['issue'], drop_first=True)\n",
    "one_hot4 = pd.get_dummies(complaints_df3['subissue'], drop_first=True)\n",
    "one_hot5 = pd.get_dummies(complaints_df3['state'], drop_first=True)\n",
    "one_hot6 = pd.get_dummies(complaints_df3['zip_code'], drop_first=True)\n",
    "one_hot7 = pd.get_dummies(complaints_df3['tags'], drop_first=True)\n",
    "\n",
    "complaints_df3 = complaints_df3.join(one_hot1, rsuffix='prod')\n",
    "complaints_df3 = complaints_df3.join(one_hot2, rsuffix='subprod')\n",
    "complaints_df3 = complaints_df3.join(one_hot3, rsuffix='issue')\n",
    "complaints_df3 = complaints_df3.join(one_hot4, rsuffix='subissue')\n",
    "complaints_df3 = complaints_df3.join(one_hot5)\n",
    "#complaints_df2 = complaints_df2.join(one_hot6)\n",
    "complaints_df3 = complaints_df3.join(one_hot7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lister(l):\n",
    "    l = literal_eval(l)\n",
    "    double_list = [l]\n",
    "    return l\n",
    "def check(l):\n",
    "    return any(isinstance(el, list) for el in l)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "complaints_df3.dropna(subset=['entities'],inplace=True)\n",
    "complaints_df3['entities2'] = complaints_df3['entities'].apply(lister)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "variable = ['company_response_to_consumer',\n",
    "            'product',\n",
    "            'subproduct',\n",
    "            'issue',\n",
    "            'subissue',\n",
    "            'state',\n",
    "            'zip_code',\n",
    "            'tags',\n",
    "            'entities']\n",
    "\n",
    "X = complaints_df3.drop(columns=variable)\n",
    "y = complaints_df3['company_response_to_consumer']\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=11)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['! II', '#', '# #', ...,\n",
       "       '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Student Aid Feedback XXXX To XXXX XX/XX/XXXX',\n",
       "       '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ XXXX XXXX XXXX To XXXX XX/XX/XXXX',\n",
       "       '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Apparently'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#df['entities'] = df['entities'].apply(lister)\n",
    "entities = mlb.fit(X['entities2'])\n",
    "mlb.classes_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_ent = mlb.transform(X_train['entities2'])\n",
    "X_test_ent = mlb.transform(X_test['entities2'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Just entities\n",
    "### Random forest"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Just with the entity data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "parameters = {#'class_weight':[None,'balanced'],\n",
    "              #'max_depth':[2,4,6,8],\n",
    "              'n_estimators':[10,20,30],\n",
    "              'n_jobs':[-1]\n",
    "             }\n",
    "rf = RandomForestClassifier()\n",
    "clf = GridSearchCV(rf, parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training accuracy: 0.9795 \n",
      "test accuracy: 0.8002 \n",
      "multiclass aucroc: 0.5144 \n",
      "f1 score: 0.7352 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "clf.fit(X_train_ent, y_train)\n",
    "y_pred = clf.predict(X_train_ent)\n",
    "y_pred_test = clf.predict(X_test_ent)\n",
    "y_score_test = clf.predict_proba(X_test_ent)[:,1]\n",
    "print(\"training accuracy:\", round(metrics.accuracy_score(y_train,y_pred),4), \"\\n\"\n",
    "     \"test accuracy:\", round(metrics.accuracy_score(y_test,y_pred_test),4), \"\\n\"\n",
    "     \"multiclass aucroc:\", round(multiclass_roc_auc_score(y_test,y_pred_test),4), '\\n'\n",
    "     \"f1 score:\",round(f1_score(y_test, y_pred_test, average='weighted'),4),'\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Naive bayes\n",
    "\n",
    "### Just with the entity data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training accuracy: 0.8152 \n",
      "test accuracy: 0.7831 \n",
      "multiclass aucroc: 0.5318 \n",
      "f1 score: 0.741 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "nb.fit(X_train_ent, y_train)\n",
    "y_pred = nb.predict(X_train_ent)\n",
    "y_pred_test = nb.predict(X_test_ent)\n",
    "y_score_test = nb.predict_proba(X_test_ent)[:,1]\n",
    "print(\"training accuracy:\", round(metrics.accuracy_score(y_train,y_pred),4), \"\\n\"\n",
    "     \"test accuracy:\", round(metrics.accuracy_score(y_test,y_pred_test),4), \"\\n\"\n",
    "     \"multiclass aucroc:\", round(multiclass_roc_auc_score(y_test,y_pred_test),4), '\\n'\n",
    "     \"f1 score:\",round(f1_score(y_test, y_pred_test, average='weighted'),4),'\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Using both sentiment and meta data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "variables_to_drop = ['complaint_id',\n",
    "                     'date_received',\n",
    "                     'consumer_complaint_narrative',\n",
    "                     'company_public_response',\n",
    "                     'company_name',\n",
    "                    'consumer_consent_provided',\n",
    "                     'submitted_via',\n",
    "                     'date_sent_to_company',\n",
    "                     'timely_response','consumer_disputed',\n",
    "                    'entity_types',\n",
    "                     'entity_sentiment_scores',\n",
    "                     'entity_sentiment_magnitudes',\n",
    "                     'entities2']\n",
    "new_x = X.drop(columns=variables_to_drop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(new_x, y, test_size=0.2, random_state=11)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training accuracy: 0.9265 \n",
      "test accuracy: 0.7529 \n",
      "multiclass aucroc: 0.5251 \n",
      "f1 score: 0.7218 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "clf.fit(X_train,y_train)\n",
    "y_pred = clf.predict(X_train)\n",
    "y_pred_test = clf.predict(X_test)\n",
    "y_score_test = clf.predict_proba(X_test)[:,1]\n",
    "print(\"training accuracy:\", round(metrics.accuracy_score(y_train,y_pred),4), \"\\n\"\n",
    "     \"test accuracy:\", round(metrics.accuracy_score(y_test,y_pred_test),4), \"\\n\"\n",
    "     \"multiclass aucroc:\", round(multiclass_roc_auc_score(y_test,y_pred_test),4), '\\n'\n",
    "     \"f1 score:\",round(f1_score(y_test, y_pred_test, average='weighted'),4),'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training accuracy: 0.8152 \n",
      "test accuracy: 0.7831 \n",
      "multiclass aucroc: 0.5318 \n",
      "f1 score: 0.741 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "nb.fit(X_train_ent, y_train)\n",
    "y_pred = nb.predict(X_train_ent)\n",
    "y_pred_test = nb.predict(X_test_ent)\n",
    "y_score_test = nb.predict_proba(X_test_ent)[:,1]\n",
    "print(\"training accuracy:\", round(metrics.accuracy_score(y_train,y_pred),4), \"\\n\"\n",
    "     \"test accuracy:\", round(metrics.accuracy_score(y_test,y_pred_test),4), \"\\n\"\n",
    "     \"multiclass aucroc:\", round(multiclass_roc_auc_score(y_test,y_pred_test),4), '\\n'\n",
    "     \"f1 score:\",round(f1_score(y_test, y_pred_test, average='weighted'),4),'\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Entites with meta data now"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train_nlp = scipy.sparse.hstack([X_train_ent, X_train])\n",
    "x_test_nlp = scipy.sparse.hstack([X_test_ent, X_test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training accuracy: 0.9822 \n",
      "test accuracy: 0.7995 \n",
      "multiclass aucroc: 0.5144 \n",
      "f1 score: 0.7348 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "clf.fit(x_train_nlp,y_train)\n",
    "y_pred = clf.predict(x_train_nlp)\n",
    "y_pred_test = clf.predict(x_test_nlp)\n",
    "y_score_test = clf.predict_proba(x_test_nlp)[:,1]\n",
    "print(\"training accuracy:\", round(metrics.accuracy_score(y_train,y_pred),4), \"\\n\"\n",
    "     \"test accuracy:\", round(metrics.accuracy_score(y_test,y_pred_test),4), \"\\n\"\n",
    "     \"multiclass aucroc:\", round(multiclass_roc_auc_score(y_test,y_pred_test),4), '\\n'\n",
    "     \"f1 score:\",round(f1_score(y_test, y_pred_test, average='weighted'),4),'\\n')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
